package com.example.book.crawler.pageprocessor;

import com.example.book.crawler.entity.CrawlerCatalog;
import com.example.book.crawler.entity.CrawlerContent;
import org.apache.commons.lang3.StringUtils;
import org.springframework.stereotype.Service;
import us.codecraft.webmagic.*;
import us.codecraft.webmagic.pipeline.Pipeline;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.UUID;

public class CatalogPageProcessor implements PageProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(1000).setTimeOut(10000)
            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36")
            .setDomain("www.google.com");

    @Override
    public void process(Page page) {

        List<CrawlerContent> list = new ArrayList<>();

        CrawlerContent content = null;
        boolean a = true;
        int i = 1;
        while (a) {
            String catalog = page.getHtml().xpath("/html/body/div/div[6]/div[1]/div[2]/ul/li[" + i + "]/a/text()").toString();
            String href = page.getHtml().xpath("/html/body/div/div[6]/div[1]/div[2]/ul/li[" + i + "]/a/@href").toString();

            if (StringUtils.isBlank(catalog)) {
                a = false;
                continue;
            }

            content = new CrawlerContent();
            content.setCatalogName(catalog);
            content.setContentHref(href);
            content.setCatalogIndex(i);
            list.add(content);
            page.putField("list", list);
            i++;
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        Spider spider = new Spider(new CatalogPageProcessor());
        spider.setUUID(UUID.randomUUID().toString());
        spider.addUrl(new String[]{"https://www.wmtxt.com/29/29276/"});
        spider.addPipeline(new Pipeline() {
            @Override
            public void process(ResultItems resultItems, Task task) {
                List<CrawlerContent> list = (List<CrawlerContent>) resultItems.get("list");
                System.out.println(">>>>" + list.size());
            }
        });
        spider.thread(1);
        spider.run();
    }
}
